{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Tuning a Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0xa45deb8>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEACAYAAABfxaZOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGYtJREFUeJzt3W+MZNV95vHvA7OAndjMyBHTmxmbxovBgxWnlyTDrpwo\nPbGDjRMNfhEhnH/uYOUN7MLGisUM+2J23wQTaWVvlBDJCpkeIhyM2VgQieABMTeRoxhw4omJh+CJ\nosbDxNNZhz+7xDJi1r99Ubfpqq6609W3/pxb5z4faTR9T53b53equk/dfurWLUUEZmbWDuelLsDM\nzKbHi76ZWYt40TczaxEv+mZmLeJF38ysRbzom5m1yKaLvqR7JK1K+npX229LelbScUn/S9Jbu247\nKOlkefu1Xe1XS/q6pG9K+sz4p2JmZpsZ5kj/MPDBDW1HgfdExAJwEjgIIOkq4AZgD3AdcLcklfv8\nPvDxiLgCuELSxu9pZmYTtumiHxFfBl7a0PZ4RHy/3PwKsLv8ej9wf0ScjYgVOk8IeyXNAW+JiKfL\nfvcCHxlD/WZmtgXjyPRvAh4pv94FnOq67XTZtgt4oav9hbLNzMymaKRFX9J/BV6PiD8eUz1mZjZB\n2+ruKGkJ+DDwM13Np4G3d23vLtuq2qu+ty8IZGZWQ0ToXLcPe6Sv8l9nQ/oQ8Elgf0S81tXvYeBG\nSRdIugy4HHgqIs4Ar0jaW76w+6vAQ5sUnu2/Q4cOJa/Bc/P8PL+8/h06dGioxXzTI31JnwMWgbdJ\n+hZwCLgDuAB4rDw55ysRcXNEnJD0AHACeB24OSLWjtpvAZaBi4BHIuLRoSrM0MrKSuoSJibnuYHn\nN+tynt+wc9t00Y+IXxzQfPgc/e8E7hzQ/tfAjwxVlZmZTYTfkZvA0tJS6hImJue5gec363Ke37Bz\n03r60hySool1mZk1mSRiTC/k2hgVRZG6hInJeW7g+c26nOc37Ny86JuZtYjjHTOzTDjeMTOzHl70\nE3CuOLs8v9mW8/yc6ZuZWR9n+mZmmXCmb2ZmPbzoJ+BccXZ5frMt5/k50zczsz7O9M3MMuFM38zM\nenjRT8C54uzy/GZbzvNzpm9mZn2c6ZuZZcKZvpmZ9fCin4Bzxdnl+c22nOfnTN/MzPo40zczy4Qz\nfTMz6+FFPwHnirPL85ttOc/Pmb6ZmfVxpm9mlgln+mZm1sOLfgLOFWeX5zfbcp7fsHPbNtkyzMym\nb25untXV5/vad+zYyYsvnklQUXNsmulLugf4eWA1It5btu0APg9cCqwAN0TEK+VtB4GbgLPAbRFx\ntGy/GlgGLgIeiYj/co4xnembWW2SgEFriMh5bRlXpn8Y+OCGtgPA4xFxJfAEcLAc8CrgBmAPcB1w\ntzr3PsDvAx+PiCuAKyRt/J5mZjZhmy76EfFl4KUNzdcDR8qvjwAfKb/eD9wfEWcjYgU4CeyVNAe8\nJSKeLvvd27VP6zhXnF2enzXVpM/TvyQiVgEi4gxwSdm+CzjV1e902bYLeKGr/YWyzczMpmhcZ+/k\nG5JNwOLiYuoSJibnuYHnZ8017GNX9+ydVUk7I2K1jG7+uWw/Dby9q9/usq2qvdLS0hLz8/MAbN++\nnYWFhTcmtfZnjLe97W1vV22vW9tuVn3j2C6KguXlZYA31stNRcSm/4B54Jmu7buA28uvbwc+VX59\nFfA14ALgMuAfWD9D6CvAXkDAI8CHzjFe5OzYsWOpS5iYnOcW4fnNCiAgBvzLd205duzY2vzOuZ5v\neqQv6XN0niLfJulbwCHgU8AXJN0EPE/njB0i4oSkB4ATwOvAzRFvnB91C72nbD463NOSmZmNi6+9\nY2bZ8Xn61XwZBjOzFvGin0D/C035yHlu4PlZcw372HnRNzNrEWf6LVV1QaqdOy/lzJmV6RdkNkbO\n9M/Rp4l3gBf9yWvrL4W1Q1t/vv1CbkPlnJvmPDfw/Ky5nOmbmVkfxzst1dY/f60d2vrz7XjHzMx6\neNFPIOfcNOe5gednzeVM394wNzePpJ5/ZtZOzvRbYHC+2c7M09rBmX41H+mbmbWIF/0Ecs5Nc54b\neH7WXM70zcysjzP9FnCmb23jTL+aj/TNzFrEi34COeemOc8NPD9rrmEfu00/I9dG40sYm1mTONOf\nsCZki870rW2a8HuXgjN9MzPr4UU/iSJ1AROTeybs+VlT+Tx9MzPr40x/wpqQLTrTt7Zpwu9dCs70\nzcyshxf9JIrUBUxM7pmw52dN5Uw/c4OukT83N5+6LDNrOGf6EzapbLEqpx/0PZ3pW9s406/mI30z\nsxYZadGX9BuS/k7S1yXdJ+kCSTskHZX0nKQvSbq4q/9BSSclPSvp2tHLn1VF6gImJvdM2POzppp4\npi/ph4H/DFwdEe+lcx2fjwIHgMcj4krgCeBg2f8q4AZgD3AdcLf8Ya1jdmFfzj+Ou9ivH5jlo3am\nXy76fwUsAP8X+BPgd4DfBX46IlYlzQFFRLxb0gEgIuKucv8/A/5bRDw54Hu3MtPfysXZtprTj5Lp\nb+X1A7MmcKZfrfaRfkT8E/A/gG8Bp4FXIuJxYGdErJZ9zgCXlLvsAk51fYvTZVs2Bh0Rb0VnwY++\nf4OeCMzM6qh9aWVJ24HrgUuBV4AvSPol+p9eaz2tLi0tMT8/D8D27dtZWFhgcXERWM+umra9vmgX\n5SwW6RxRd28DfKZnrv1Z3Mb+nT4bx+vvX71/p21xw+2D+2+c32b7d/fvri314zGJbc9vdrbXrW03\nq75xPV7Ly8tsxSjxzi8AH4yIXy+3fwX4D8DPAItd8c6xiNgzIN55FDiUU7wzfORSAPuGjFE632PY\nyCV1vNP95JQjz282tDHeKYqCffv2bRrvjLLo7wXuAX4CeA04DDwNvAN4MSLuknQ7sCMiDpQv5N4H\nXEMn1nkMeNeg1T3/Rb/TnuOib9YEbVz0YbhMv3a8ExFPSXoQ+Brwevn/Z4G3AA9Iugl4ns4ZO0TE\nCUkPACfK/jfP5MqevQvHcsaPmTXTSOfpR8R/j4g9EfHeiPhYRLweES9GxAci4sqIuDYiXu7qf2dE\nXF7uc3T08mdVkbqAc3iN/heTh9efp+bF87OmGvax8ztyzcxaxNfeGaNcMv3h+uadjdpsc6ZfzUf6\nZmYt4kU/iSJ1AROTeybs+VlTOdM3M7M+zvTHyJm+WTM406/mI30zsxbxop9EkbqAick9E/b8rKmc\n6ZuZWR9n+mPkTN+sGZzpV/ORvplZi3jRT6LYYv/+j0FsqtwzYc/PmmrYx672VTZtVFu5muXaRdC6\nNXfhN7PmcqY/Rk3I2ac31kV0nox6Dfo8X7Npc6Z/jj5NvAO86Nft24yxZvGxs7x40a/mTD+JInUB\nVlPumXfu88uZz9M3M7M+jnfGyPFOp30WHzvLi+Odaj7S38Tc3Hzf6ZKSmJubT12amdmWedHfxOrq\n8/R/ZmyU7XUVY6jMUsg98859fjlzpm9mZn2c6W9i1q6H04Sxhn3s5ubm+/5i8nn+Ng7O9M/Rp4l3\ngBf9un2bMdawj13V/dWUx95mlxf9ao53kihSF2A15Z555z6/nDnTNzOzPo53NuF4Z+v7O96x1Bzv\nVPORvplZi3jRT6JIXYDVlHvmnfv8cjaVTF/SxZK+IOlZSd+QdI2kHZKOSnpO0pckXdzV/6Ckk2X/\na0cZ22ZH1buazWz6Rsr0JS0Dfx4RhyVtA34AuAP4l4j4bUm3Azsi4oCkq4D7gJ8AdgOPA+8aFN47\n06/btxljbeWzf53p2yQ4069W+0hf0luBn4qIwwARcTYiXgGuB46U3Y4AHym/3g/cX/ZbAU4Ce+uO\nb2ZmWzdKvHMZ8B1JhyX9jaTPSnozsDMiVgEi4gxwSdl/F3Cqa//TZVsLFakLsJpyz7xzn1/OpvEZ\nuduAq4FbIuKrkj4NHKD/b6paf0stLS0xPz8PwPbt21lYWGBxcRFYn9y0ttcX6Y3b9PTvaqnov7Z9\nvKJ/1f5rbYPHH9/+G2+vt3///Tfa/t72dp3tdWvbzapvHNtFUbC8vMxW1M70Je0E/ioi3llu/ySd\nRf/fAYsRsSppDjgWEXskHQAiIu4q+z8KHIqIJwd8b2f6tfo2Yyxn+paaM/1qteOdMsI5JemKsun9\nwDeAh4Glsu1jwEPl1w8DN0q6QNJlwOXAU3XHNzOzrRv1PP1bgfskHQd+FPgt4C7gZyU9R+eJ4FMA\nEXECeAA4ATwC3NyYw/mpK1IXYDXlnnnnPr+cTSPTJyL+ls4pmBt9oKL/ncCdo4xpubtw4Dn8vuSy\n2Xj42jubcKa/9f1HzfTbmMXaeDnTr+bLMJiZtYgX/SSK1AVYTbln3rnPL2fDPnZe9M3MWsSZ/iac\n6W99f2f6lpoz/Wojnb3TboPPMjEzazLHO7W9RudIovvfsIpJFGRTkHvmnfv8cuZMv4ZB1303M8uJ\nM/3ecck1Z3emb23iTL+aj/TNzFrEi34SReoCrKbcM+/c55czZ/pmZtbHmX7vuOSas89+pn8RnTOm\n1vkibFbFmX41n6dvYzap9y+snSK7bnXVZ1eZbZXjnSSK1AVM0CjvX2i+3DPv3OeXM2f6ZmbWx5l+\n77jkmrPP9ljVfZv482vpOdOv5iN9M7MW8aKfRJG6AKsp98w79/nlzJm+mZn1cabfOy6znX3nOlZ1\n3yb+/Fp6zvSr+UjfWmHQFVTn5uZTl2U2dV70kyhSF9A6q6vPs/H9A522rck98859fjlzpm9mZn2c\n6feOy2xn37mOVd132J+Tqse2iT//Njpn+tV8pG9ZGZTd+xPQzNZ50U+iSF1AJi7sW9wHZffjvP5P\n7pl37vPL2bCPna+yaTOs/8qbnRjIzKqMnOlLOg/4KvBCROyXtAP4PHApsALcEBGvlH0PAjcBZ4Hb\nIuJoxfd0pu+xplJXzvlumznTrzaOeOc24ETX9gHg8Yi4EngCOFgWcxVwA7AHuA64Ww5bzcymaqRF\nX9Ju4MPAH3Q1Xw8cKb8+Anyk/Ho/cH9EnI2IFeAksHeU8WdXkboAqyn3zDv3+eVsWufpfxr4JL1/\nR+2MiFWAiDgDXFK27wJOdfU7XbaZmdmU1H4hV9LPAasRcVzS4jm61grQlpaWmJ+fB2D79u0sLCyw\nuNgZZu0Zbdzb69a2FzfZ3th/2P032mz/tba64w+7/8bbm7J/Vf/x7L+Vn5fFxcWJ/fw1YTun+a1b\n225WfePYLoqC5eVlgDfWy83UfiFX0m8Bv0znRdk3AW8Bvgj8OLAYEauS5oBjEbFH0gEgIuKucv9H\ngUMR8eSA7+0Xcj3WVOrK+UW9NvMLudVqxzsRcUdEvCMi3gncCDwREb8C/CmwVHb7GPBQ+fXDwI2S\nLpB0GXA58FTd8WdbkboAqyn3zDv3+eUs5Xn6nwIekHQT8DydM3aIiBOSHqBzps/rwM1JDufNzFrM\n197pHZfUcYPHcrxjo3O8U82XYTAzaxEv+kkUqQuwmnLPvHOfX858PX0zM+vjTL93XFJnzB7Lmb6N\nzpl+NR/pm5m1iBf9JIrUBVhNuWfeuc8vZ870zcysjzP93nFJnTF7LGf6Njpn+tV8pG8t1v9xi5KY\nm5tPXZjZxHjRT6JIXYAB6x+32Puv8zm7g+Weeec+v5w50zerrf8vAB/9Wy6c6feOS+qM2WM1t64m\n/q7YYM70q/lI38ysRbzoJ1GkLsBqyj3zzn1+OXOmb2ZmfZzp947LLGXM7RmrGXU18XfFBnOmX81H\n+mZmLeJFP4kidQFW0zQy77m5+WRvGnOmP7tSfkaumY2g8+aw/ghidfWcf7WbDcWZfu+4zFLG3J6x\nmlHXtH4m25pHj1Nb70Nn+mZm1sOLfhJF6gKsptwz79znlzOfp38OVS+UmZnlrpWZ/rnyvlnKmNsz\nVjPqmsTP5NzcfMVVPduXR4+TM/1qPnvHbAqqF3cY/GRkNhmtjHfSK1IXYDXVzbzXT8Pc+K9ZnOnP\nLmf6ZmbWx5l+7y0D2pubMbdnrGbUNezP5FZz+q3U1cTf1yZypl+t9pG+pN2SnpD0DUnPSLq1bN8h\n6aik5yR9SdLFXfsclHRS0rOSrq07tlkTVJ0FNjjKMWuGUeKds8AnIuI9wH8EbpH0buAA8HhEXAk8\nARwEkHQVcAOwB7gOuFutPU+ySF2A1dSdm85KTr8VzvRn18SvvRMRZ4Az5devSnoW2A1cD/x02e0I\nnRXuALAfuD8izgIrkk4Ce4En69ZgNj0X+r0cloWxvJAraR5YAL4C7IyIVXjjieGSstsu4FTXbqfL\nthZaTF2Abdlr5HREX2VxcTF1CVbTsI/dyIu+pB8EHgRui4hX6f9tyPO3w8xsBo305ixJ2+gs+H8U\nEQ+VzauSdkbEqqQ54J/L9tPA27t23122DbS0tMT8/DwA27dvZ2Fh4Y1nsrXsqu52R8H6EXexYfS1\n7Y23b9Z/2P0/s8X919rqjj/s/htvb8r+Vf0ntf9a26D9N+671f0H9d/a/qP+/J9ruzsXnsT3n+b2\nurXtZtU3rsdreXmZrRjplE1J9wLfiYhPdLXdBbwYEXdJuh3YEREHyhdy7wOuoRPrPAa8a9C5mfmf\nslkA+6Y01qT6tu+UzU5bwfpiPO374CI6MdO6nTsv5cyZlQF96ymKIouIp42nbBZFwb59+zY9ZbP2\noi/pfcBfAM+wHnLeATwFPEDnqP554IaIeLnc5yDwceB1OnHQ0Yrvnfmi77HyqKsZ90Gui9go2rjo\nw3Dn6fvNWb23DGifpYUh17GaWlcz7oMm/g6n5kW/mi/DkESRugCrrUhdwET5PP3Z5WvvmJlZH8c7\nvbcMaJ+lCCDXsZpaVzPugyb+DqfmeKeaj/TNZtqFfdf+mZubT12UNZgX/SSK1AVYbUXqAjbof6dw\n9Ye1bM6Z/uxypm/WWv1H//4LwNY40++9ZUD7LOW+uY7V1Lpm7z5o4u/7JDjTr+YjfTOzFvGin0SR\nugCrrUhdwFhVfRCMo6DZ40zfzDbV/0Ewxxj1xWBrNmf6vbcMaJ+l3DfXsZpa1+zdBxt/r3LNvnOd\n12aGyfRHurSymc0Sf/qXOd5JpEhdgNVWpC5gBMN8+lcxzYJsjJzpm9kIxv9OX79o3AzO9HtvGdA+\nS7lvrmM1ta623QejfYjLNHN2Z/rVnOmb2ZDW4qF1q6t+jWDWON5JokhdgNVWpC5gwoqxfJdBUY5N\n1rCZvo/0zWzs1s//7+aFvwmc6ffeMqC9qflqm8Zqal2+Dwbl/Ovqv39gVM70q/lI38xG0J/zd/io\nvqmc6SdRpC7AaitSFzBhReoCrCafp1/yC0pmTedP/5qm7DP9wdlervlqrmM1tS7fB5Mca5Q1wJl+\nteyP9M0sD35H73h40U+iSF2A1VakLmDCitQFlPojn/7LQHf+ra6ecYSLz9M3s5k26KygqsV8K33N\nmf6m7c3NPNszVlPr8n0wi2M1cc0bF2f6ZmbWY+qLvqQPSfp7Sd+UdPu0x2+GInUBVluRuoAJK1IX\nYDU18jx9SecBvwt8EHgP8FFJ755mDc1wPHUBVlvuj13u88vX8ePDPXbTfiF3L3AyIp4HkHQ/cD3w\n98N+g1dffZUHH3ywr/0Tn7iDl1769rjqnLCXUxdgteX+2OU+v3y9/PJwj920F/1dwKmu7RfoPBEM\nbXl5md/8zd9j27Zrulq/z7/+67fxNUDMzM5t5k7ZPP/885G+x/nn/0tX6/eT1VPPSuoCrLaV1AVM\n2ErqAqymlZWVofpNe9E/Dbyja3t32dZnszdYfO97K4P2qug9qH2afT1WXnUdmeJYdfuOsv+RivZJ\njLXVvqOP1dY3b62Z6nn6ks4HngPeD3wbeAr4aEQ8O7UizMxabKpH+hHx/yT9J+AonTOH7vGCb2Y2\nPY18R66ZmU1Go96Rm/MbtyTdI2lV0tdT1zIJknZLekLSNyQ9I+nW1DWNk6QLJT0p6Wvl/A6lrmnc\nJJ0n6W8kPZy6lnGTtCLpb8vH76nU9YybpIslfUHSs+Xv4DWVfZtypF++ceubdPL+fwKeBm6MiKHP\n4W8yST8JvArcGxHvTV3PuEmaA+Yi4rikHwT+Grg+l8cPQNKbI+K75WtTfwncGhHZLCCSfgP4MeCt\nEbE/dT3jJOkfgR+LiJdS1zIJkpaBP4+Iw5K2AW+OiP8zqG+TjvTfeONWRLwOrL1xKwsR8WUgyx84\ngIg4ExHHy69fBZ6l876MbETEd8svL6TzelgzjpjGQNJu4MPAH6SuZUJEs9a7sZH0VuCnIuIwQESc\nrVrwoVl3wqA3bmW1aLSFpHlgAXgybSXjVcYfXwPOAI9FxNOpaxqjTwOfJKMnsg0CeEzS05J+PXUx\nY3YZ8B1Jh8t47rOS3lTVuUmLvmWgjHYeBG4rj/izERHfj4h/T+f9JddIuip1TeMg6eeA1fIvNZHn\n29jfFxFX0/lr5pYybs3FNuBq4PfKOX4XOFDVuUmL/tBv3LJmKrPEB4E/ioiHUtczKeWfzseAD6Wu\nZUzeB+wvc+8/BvZJujdxTWMVEd8u///fwBfZ4uVfGu4F4FREfLXcfpDOk8BATVr0nwYul3SppAuA\nG4HcziLI9ShqzR8CJyLif6YuZNwk/ZCki8uv3wT8LFu4UGCTRcQdEfGOiHgnnd+7JyLiV1PXNS6S\n3lz+BYqkHwCuBf4ubVXjExGrwClJV5RN7wdOVPVvzLV3cn/jlqTPAYvA2yR9Czi09sJLDiS9D/gl\n4Jky9w7gjoh4NG1lY/NvgSPlWWbnAZ+PiEcS12TD2Ql8UVLQWfPui4ijiWsat1uB+yT9G+AfgV+r\n6tiYUzbNzGzymhTvmJnZhHnRNzNrES/6ZmYt4kXfzKxFvOibmbWIF30zsxbxom9m1iJe9M3MWuT/\nA5yV7vDKKik8AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xa45d9b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.datasets import fetch_california_housing\n",
    "\n",
    "cali_housing = fetch_california_housing()\n",
    "\n",
    "X = cali_housing.data\n",
    "y = cali_housing.target\n",
    "\n",
    "pd.Series((y)).hist(bins=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0xa6741d0>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEACAYAAABfxaZOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGjJJREFUeJzt3X+MVed95/H3x8GO7RQzuFmY7uB4nLjjQNR0jJtxV27W\n03UCwd3FXmtFiaq1J6b5w6aLk11VYSKt6P6xSqm0Com2thQ19UDl1CFeec02GDDCN1Wr2pAEihsw\noGZxGDZM2oaSeL2qIPnuH/cZzplhfpzrXO45M+fzkkb3Ps88h/uc72W+97nfc+49igjMzKweril7\nAmZm1jlO+mZmNeKkb2ZWI076ZmY14qRvZlYjTvpmZjVSKOlLekLSa+lnU+pbLGmfpBOS9kpalBs/\nLOmUpOOSVuX6V0o6KumkpG3t3x0zM5vJrElf0geADcCvAP3Av5b0PmAzsD8i7gAOAMNp/ApgHbAc\nWAM8KUnpn3sK2BARfUCfpNVt3h8zM5tBkZX+cuDViPiniPgJ8OfAQ8BaYHsasx14MN1fCzwbEZci\n4jRwChiQ1A0sjIhDadyO3DZmZtYBRZL+3wAfTuWcG4H7gVuApRExBhAR54AlaXwPcCa3/dnU1wOM\n5vpHU5+ZmXXIgtkGRMTrkrYCLwFvAoeBn0w1tM1zMzOzNps16QNExNPA0wCS/ivNlfyYpKURMZZK\nNz9Iw8/SfCcwblnqm67/CpL8AmJm9jZEhGb6fdGzd/5Zun0P8G+BrwC7gKE05BHghXR/F7Be0nWS\nbgNuBw6mEtAFSQPpwO7DuW2mmrh/ItiyZUvpc6jKj2PhWDgWM/8UUWilD/wPSTcDF4HHI+JHqeSz\nU9KjwBs0z9ghIo5J2gkcy40fn81GYAS4HtgdEXsKPn5tnT59uuwpVIZjkXEsMo5Fa4qWd/7lFH0/\nBD4yzfjPAZ+bov9bwC+1OEczM2sTfyK34oaGhsqeQmU4FhnHIuNYtEZF60CdJCmqOC8zsyqTRLTj\nQK6Vp9FolD2FynAsMo5FxrFojZO+mVmNuLxjZjZPuLxjZmYTOOlXnOuVGcci41hkHIvWOOmbmdWI\na/pmZvOEa/pmZjaBk37FuV6ZcSwyjkXGsWiNk76ZWY24pm9mNk+4pj8PdHf3IumKn+7u3rKnZmZz\nkJN+xY2NvUHzSpQTf5r99eLabcaxyDgWrXHSNzOrkUI1fUmfBjYAPwVeAz4BvAv4KnArcBpYFxEX\n0vhh4FHgEvBEROxL/SuZeOWsT03zeK7pJ80rS04VCxW+PJqZ1UNbavqS/jnwH4CVEfFBmlfb+jiw\nGdgfEXcAB4DhNH4FzUsnLgfWAE+ma+ICPAVsiIg+oE/S6re1Z2Zm9rYULe+8A3iXpAXADcBZ4AFg\ne/r9duDBdH8t8GxEXIqI08ApYEBSN7AwIg6lcTty25jNyrXbjGORcSxaM2vSj4j/A/w34Hs0k/2F\niNgPLI2IsTTmHLAkbdIDnMn9E2dTXw8wmusfTX1mZtYhs14YXVIXzVX9rcAF4GuSfosrC81tLTAP\nDQ3R29sLQFdXF/39/QwODgLZK3td2tBIt5PbVGJ+bpfTHleV+ZT399Hsq8p8OtluNBqMjIwAXM6X\ns5n1QK6kfwesjohPpva/B34V+FfAYESMpdLNyxGxXNJmICJiaxq/B9gCvDE+JvWvB+6NiMemeEwf\nyE18INfMimrXh7O+B/yqpOvTAdn7gGPALmAojXkEeCHd3wWsl3SdpNuA24GDqQR0QdJA+ncezm1j\nNqvJK9w6cywyjkVrZi3vRMRBSc8Bh4GL6fZLwEJgp6RHaa7i16XxxyTtpPnCcBF4PLds38jEUzb3\ntHd3zMxsJv7unYpzecfMivJ375iZ2QRO+jZnuHabcSwyjkVrnPTNzGrENf2Kc03fzIpyTd/MzCZw\n0rc5w7XbjGORcSxa46RvZlYjrulXnGv6ZlaUa/pmZjaBk77NGa7dZhyLjGPRGid9M7MacU2/4lzT\nN7OiXNM3M7MJnPRtznDtNuNYZByL1jjpm5nViGv6FeeavpkV1ZaavqQ+SYclfTvdXpC0SdJiSfsk\nnZC0V9Ki3DbDkk5JOi5pVa5/paSjkk5K2vaz7Z6ZmbVq1qQfEScj4s6IWAncBfxf4HlgM7A/Iu4A\nDgDDAJJW0Lx04nJgDfBkuiYuwFPAhojoA/okrW73Dtn85dptxrHIOBatabWm/xHgbyPiDPAAsD31\nbwceTPfXAs9GxKWIOA2cAgYkdQMLI+JQGrcjt42ZmXVAq0n/N4GvpPtLI2IMICLOAUtSfw9wJrfN\n2dTXA4zm+kdTn1khg4ODZU+hMhyLjGPRmgVFB0q6luYq/jOpa/JRxLYeVRwaGqK3txeArq4u+vv7\nLz+542/n6tKGRrqd3KYS83PbbbfLaTcaDUZGRgAu58vZFD57R9Ja4PGI+FhqHwcGI2IslW5ejojl\nkjYDERFb07g9wBbgjfExqX89cG9EPDbFY/nsncRn72Qajcbl//h151hkHItMuz+R+3HgT3PtXcBQ\nuv8I8EKuf72k6yTdBtwOHEwloAuSBtKB3Ydz25iZWQcUWulLupHmSv29EfHj1HczsBO4Jf1uXUT8\nY/rdMLABuAg8ERH7Uv9dwAhwPbA7Ip6Y5vG80k+80jezooqs9P3hrIpz0jezovyFazavjB/AMsci\nz7FojZO+mVmNuLxTcS7vmFlRLu+YmdkETvo2Z7h2m3EsMo5Fa5z0zcxqxDX9inNN38yKck3fzMwm\ncNK3OcO124xjkXEsWuOkb2ZWI67pV5xr+mZWlGv6ZmY2gZO+zRmu3WYci4xj0RonfTOzGnFNv+Jc\n0zezolzTNzOzCQolfUmLJH1N0nFJ35F0t6TFkvZJOiFpr6RFufHDkk6l8aty/SslHZV0UtK2q7FD\nNn+5dptxLDKORWuKrvS/QPPyhsuBXwZeBzYD+yPiDuAAMAwgaQWwDlgOrAGeTNfEBXgK2BARfUCf\npNVt2xMzM5vVrDV9STcBhyPifZP6XwfujYgxSd1AIyLeL2kzEBGxNY17Efg9mtfRPRARK1L/+rT9\nY1M8pmv6iWv6ZlZUu2r6twF/L+lpSd+W9KV0ofSlETEGEBHngCVpfA9wJrf92dTXA4zm+kdTn5mZ\ndciCgmNWAhsj4puSPk+ztDN5mdnWZefQ0BC9vb0AdHV10d/fz+DgIJDV8OrShka6ndymEvPrVHu8\nryrzKbN95MgRPvWpT1VmPmW2t23bVtv80Gg0GBkZAbicL2dTpLyzFPiriHhvav8azaT/PmAwV955\nOSKWT1He2QNsoVneeTkdF3B5pyCXdzKNRiP3QlhvjkXGscgUKe8UOk9f0jeAT0bESUlbgBvTr34Y\nEVslfQZYHBGb04HcZ4C7aZZvXgJ+MSJC0ivAJuAQ8HXgixGxZ4rHc9JPnPTNrKgiSb9IeQeaifoZ\nSdcC3wU+AbwD2CnpUZqr+HUAEXFM0k7gGHAReDyXwTcCI8D1NM8GuiLhm5nZ1eNP5FacV/oZv43P\nOBYZxyLjT+SamdkEXulXnFf6ZlaUV/pmZjaBk77NGfnz9evOscg4Fq1x0jczqxHX9CvONX0zK8o1\nfTMzm8BJ3+YM124zjkXGsWiNk76ZWY24pl9xrumbWVGu6ZuZ2QRO+jZnuHabcSwyjkVrnPTNzGrE\nNf2Kc03fzIpyTd/MzCYolPQlnZb015IOSzqY+hZL2ifphKS9khblxg9LOiXpuKRVuf6Vko5KOilp\nW/t3x+Yz124zjkXGsWhN0ZX+T2leD/fOiBhIfZuB/RFxB3AAGAZIl0tcBywH1gBPqlmjAHgK2BAR\nfUCfpNVt2g8zMyug6DVy/zfwKxHxD7m+12le2Hz8wuiNiHj/FBdGfxH4PZqXVDwQEStSvy+MXoBr\n+mZWVDtr+gG8JOmQpN9OfUsjYgwgIs4BS1J/D3Amt+3Z1NcDjOb6R1OfmZl1SNGkf09ErATuBzZK\n+jBXLj+97LSryrXbjGORcSxas6DIoIj4frr9O0n/ExgAxiQtzZV3fpCGnwVuyW2+LPVN1z+loaEh\nent7Aejq6qK/v//yxY/Hn+S6tKGRbie3qcT8OtWu2/7O1D5y5Eil5lNm+8iRI5WaTyfbjUaDkZER\ngMv5cjaz1vQl3QhcExFvSnoXsA/4L8B9wA8jYqukzwCLI2JzOpD7DHA3zfLNS8AvRkRIegXYBBwC\nvg58MSL2TPGYruknrumbWVFFavpFVvpLgeclRRr/TETsk/RNYKekR2kepF0HEBHHJO0EjgEXgcdz\nGXwjMAJcD+yeKuGbmdnV40/kVpxX+pmbb+7m/PmxK/qXLr2Vc+dOd35CJWo0Gpff7tedY5Fp10rf\nrBKaCf/KF7qxsRn/j5tZjlf6FeeVfsaxMJuZv3vHzMwmcNI3m4Mmn8ZaZ45Fa5z0zcxqxDX9inMd\nO+NYmM3MNX0zM5vASd9sDnIdO+NYtMZJ38ysRlzTrzjXsTOOhdnMXNM3M7MJnPTN5iDXsTOORWuc\n9M3MasQ1/YpzHTvjWJjNzDV9MzObwEnfbA5yHTvjWLSmcNKXdI2kb0valdqLJe2TdELSXkmLcmOH\nJZ2SdFzSqlz/SklHJZ2UtK29u2JmZrMpXNOX9GngLuCmiFgraSvwDxHxB9NcI/dDNC9+vp/sGrmv\nAr8TEYck7Qa+EBF7p3gs1/QT17EzjoXZzNpW05e0DLgf+KNc9wPA9nR/O/Bgur8WeDYiLkXEaeAU\nMCCpG1gYEYfSuB25bczMrAOKlnc+D/wuE5dZSyNiDCAizgFLUn8PcCY37mzq6wFGc/2jqc/MWuQ6\ndsaxaM2sSV/SbwBjEXEEmOltg99fm5lVXJELo98DrJV0P3ADsFDSnwDnJC2NiLFUuvlBGn8WuCW3\n/bLUN13/lIaGhujt7QWgq6uL/v7+y1e8H39lr0sbGul2cptKzK9T7dwep9vBrKfRKH1+ZcWjKvMp\n7++jns//4OAgjUaDkZERgMv5cjYtfThL0r3Af0oHcv+A5oHcrdMcyL2bZvnmJbIDua8Am4BDwNeB\nL0bEnikexwdyEx+8zDgWZjO72h/O+n3go5JOAPelNhFxDNgJHAN2A4/nMvhG4MvASeDUVAnfzGZ3\n5buf+nIsWlOkvHNZRHwD+Ea6/0PgI9OM+xzwuSn6vwX8UuvTNDOzdvB371ScSxoZx8JsZv7uHTMz\nm8BJ32wOch0741i0xknfzKxGXNOvONexM46F2cyK1PRbOnunk9797luv6PvQh+7mxRd3ljAbM7P5\nobIrfTg9qfccXV0Pcf78tB/inZe8us04Fpn8J1DrzrHIzOmVPkxe6Vd4qmZmc0SFV/qT53WWrq4B\nr/Sz39RudetYmM3M5+mbmdkETvpmc5DPTc84Fq1x0jczqxHX9CvOdeyMY2E2M9f0zcxsAid9sznI\ndeyMY9EaJ30zsxopcmH0d0p6VdJhSa9J2pL6F0vaJ+mEpL2SFuW2GZZ0StJxSaty/SslHZV0UtK2\nq7NLZvOfP4GacSxaM2vSj4h/An49Iu4E+oE1kgaAzcD+iLgDOAAMA6Rr5K4DlgNrgCfVPAIH8BSw\nISL6gD5Jq9u9Q2ZmNr1C5Z2IeCvdfSfN70MI4AFge+rfDjyY7q8Fno2ISxFxGjgFDEjqBhZGxKE0\nbkduGzNrgevYGceiNYWSvqRrJB0GzgEvpcS9NCLGACLiHLAkDe8BzuQ2P5v6eoDRXP9o6jMzsw4p\n9C1mEfFT4E5JNwHPS/oAV54w3eYTpYeA3nS/i/zrw/gr+3gtb763oZFuJ7epxPw61c7tcbodzHpy\n37RYlfl2Kh5VmU95fx/1fP4HBwdpNBqMjIwA0NvbSxEtfzhL0n8G3gJ+GxiMiLFUunk5IpZL2gxE\nRGxN4/cAW4A3xsek/vXAvRHx2BSP4Q9nJf5AUsaxMJtZWz6cJend42fmSLoB+ChwHNhFczkO8Ajw\nQrq/C1gv6TpJtwG3AwdTCeiCpIF0YPfh3DZm1oIr3/3Ul2PRmiLlnV8Atku6huaLxFcjYrekV4Cd\nkh6luYpfBxARxyTtBI4BF4HHc9c+3AiMANcDuyNiT1v3xszMZuTv3qk4lzQyjoXZzPzdO2ZmNoGT\nvtkc5Dp2xrFojZO+mVmNuKZfca5jZxwLs5m5pm9mZhM46ZvNQa5jZxyL1jjpm5nViGv6Fec6dsax\nMJuZa/pmZjaBk77ZHOQ6dsaxaI2TvplZjbimX3GuY2ccC7OZuaZvZmYTOOmbzUGuY2cci9Y46ZuZ\n1Yhr+hXnOnbGsTCbWbsul7hM0gFJ35H0mqRNqX+xpH2STkjaO35JxfS7YUmnJB2XtCrXv1LSUUkn\nJW37WXbOzMxaV6S8cwn4jxHxAeBfABslvR/YDOyPiDuAA8AwgKQVNC+duBxYAzyZrokL8BSwISL6\ngD5Jq9u6N2Y14Tp2xrFozaxJPyLORcSRdP9NmhdFXwY8AGxPw7YDD6b7a4FnI+JSRJwGTgEDkrqB\nhRFxKI3bkdvGzFrw0EPrkXTFT3d3b9lTs4orcmH0yyT1Av3AK8DSiBiD5guDpCVpWA/wV7nNzqa+\nS8Born809ZtZi86fH2Oq4xtjYzOWc+elwcHBsqcwpxRO+pJ+DngOeCIi3mwebJ2gzUfShoDedL+L\n/OvD+Nu58Sd7vrehkW4nt6nE/DrVzu1xuh3MehqN0udXlXhUZX5uX/12o9FgZGQEgN7eXooodPaO\npAXAnwEvRsQXUt9xYDAixlLp5uWIWC5pMxARsTWN2wNsAd4YH5P61wP3RsRjUzyez95JfMZKxrHI\nOBaZ/At+3bXzE7l/DBwbT/jJLprLcYBHgBdy/eslXSfpNuB24GBEnAMuSBpIB3Yfzm1jZmYdMOtK\nX9I9wJ8Dr9FcWgTwWeAgsBO4heYqfl1E/GPaZhjYAFykWQ7al/rvAkaA64HdEfHENI/plX7iFV3G\nscg4FjaVIit9fzir4vzHnXEsMo6FTcVfuGZm896VB7ZtJk76ZmY14qRvZnOaz9xpjZO+mVmNOOmb\n2Zzmmn5rnPTNzGrESd/M5jTX9FvjpG9mViNO+mY2p7mm3xonfTOzGnHSN7M5zTX91jjpm5nViJO+\nmc1prum3xknfzKxGnPTNbE5zTb81TvpmZjUya9KX9GVJY5KO5voWS9on6YSkvZIW5X43LOmUpOOS\nVuX6V0o6KumkpG3t3xUzqyPX9FtTZKX/NLB6Ut9mYH9E3AEcAIYBJK0A1gHLgTXAk+l6uABPARsi\nog/okzT53zQzs6ts1qQfEX8BnJ/U/QCwPd3fDjyY7q8Fno2ISxFxGjgFDEjqBhZGxKE0bkduGzOz\nt801/da83Zr+kogYA4iIc8CS1N8DnMmNO5v6eoDRXP9o6jMzsw5a0KZ/5ypciXkI6E33u8i/RozX\n8MZf4ed7GxrpdnKbSsyvU+3cHqfbwayn0Sh9flWJR1Xm16n2tm3b6O/vr8x8rnb75pu7OX9+jLdL\nEbPna0m3Av8rIj6Y2seBwYgYS6WblyNiuaTNQETE1jRuD7AFeGN8TOpfD9wbEY9N83hx5evIWbq6\nBjh//uzb29M5qnlIZKrnSBR57uYTxyLjWGTyL/h1MP1zD+n51zS/BIqXd5R+xu2iuRQHeAR4Ide/\nXtJ1km4DbgcOphLQBUkD6cDuw7ltzMzetjol/HaYtbwj6Ss03zf+vKTv0Vy5/z7wNUmP0lzFrwOI\niGOSdgLHgIvA45EtOzYCI8D1wO6I2NPeXTEzs9kUKu90mss7Gb+NzzgWGcci4/LOhN+2rbxjZlZJ\nDz20HklX/HR395Y9tUpq19k7ZmalaJ7JcuXKd2xsxgVvbXmlb2ZWI076ZmY14qRvZlYjTvpmZjXi\npG9mViNO+mZmNeKkb2ZWI076ZmY14qRvZlYjTvpmZjXipG9mViNO+mZmNeKkb2ZWIx1P+pI+Jul1\nSSclfabTj29mVmcdTfqSrgH+O7Aa+ADwcUnv7+QczMzqrNMr/QHgVES8EREXgWeBBzo8BzOz2up0\n0u8BzuTao6nPzMw6oLJXzrrppn8zoR3x/7j22mtLmo2Z2fzQ6aR/FnhPrr0s9V3hRz/6syv6fvzj\n8YsC183U++xY5Hodi6zXsch6520s3v5+KWK6q6q3n6R3ACeA+4DvAweBj0fE8Y5Nwsysxjq60o+I\nn0j6HWAfzeMJX3bCNzPrnI6u9M3MrFyV+kSupC9LGpN0tOy5lEnSMkkHJH1H0muSNpU9p7JIeqek\nVyUdTrHYUvacyibpGknflrSr7LmUSdJpSX+d/m8cLHs+ZZK0SNLXJB1PeePuacdWaaUv6deAN4Ed\nEfHBsudTFkndQHdEHJH0c8C3gAci4vWSp1YKSTdGxFvpmNBfApsiorZ/5JI+DdwF3BQRa8ueT1kk\nfRe4KyLOlz2XskkaAb4REU9LWgDcGBE/mmpspVb6EfEXQO2fwIg4FxFH0v03gePU+PMMEfFWuvtO\nmsehqrNS6TBJy4D7gT8qey4VICqWw8og6SbgwxHxNEBEXJou4YMDVnmSeoF+4NVyZ1KeVM44DJwD\nXoqIQ2XPqUSfB36XGr/w5QTwkqRDkj5Z9mRKdBvw95KeTmW/L0m6YbrBTvoVlko7zwFPpBV/LUXE\nTyPiTpqf67hb0oqy51QGSb8BjKV3geJnOVl7frgnIlbSfOezMZWH62gBsBL4wxSPt4DN0w120q+o\nVJd7DviTiHih7PlUQXrL+jLwsbLnUpJ7gLWplv2nwK9L2lHynEoTEd9Pt38HPE/zu73qaBQ4ExHf\nTO3naL4ITKmKSd8rmKY/Bo5FxBfKnkiZJL1b0qJ0/wbgo0AtD2hHxGcj4j0R8V5gPXAgIh4ue15l\nkHRjeieMpHcBq4C/KXdW5YiIMeCMpL7UdR9wbLrxlfruHUlfAQaBn5f0PWDL+MGJOpF0D/BbwGup\nlh3AZyNiT7kzK8UvANvT13JfA3w1InaXPCcr31LgeUlBM489ExH7Sp5TmTYBz0i6Fvgu8InpBlbq\nlE0zM7u6qljeMTOzq8RJ38ysRpz0zcxqxEnfzKxGnPTNzGrESd/MrEac9M3MasRJ38ysRv4/ZOwo\nuLbsGFYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x3e3b6a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "bins = np.arange(6)\n",
    "binned_y = np.digitize(y, bins)\n",
    "pd.Series(binned_y).hist(bins=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,stratify=binned_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise',\n",
       "       estimator=RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
       "           max_features='auto', max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "           min_samples_leaf=1, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "           oob_score=False, random_state=None, verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'max_features': ['sqrt', 'log2', 1.0], 'n_estimators': [50], 'min_samples_leaf': [1, 2, 7, 11]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "rft = RandomForestRegressor()\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "param_grid = {'max_features': ['sqrt','log2',1.0],\n",
    " 'min_samples_leaf': [1,2,7,11],\n",
    " 'n_estimators': [50]\n",
    " }\n",
    "\n",
    "pre_gs_inst = GridSearchCV(rft, param_grid = param_grid,cv=3) #changed from cv=10 to speed up computation\n",
    "\n",
    "pre_gs_inst.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_features': 'log2', 'min_samples_leaf': 2, 'n_estimators': 50}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pre_gs_inst.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
       "           max_features='log2', max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "           min_samples_leaf=2, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, n_estimators=1000, n_jobs=1,\n",
       "           oob_score=False, random_state=None, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rs_rft = RandomForestRegressor(**{'max_features': 'log2', 'min_samples_leaf': 2, 'n_estimators': 1000})\n",
    "\n",
    "rs_rft.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.82099515511365539"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rs_rft.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.82099515511365539"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = rs_rft.predict(X_test)\n",
    "\n",
    "from sklearn.metrics import r2_score\n",
    "r2_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.32253091494598596"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import mean_absolute_error\n",
    "mean_absolute_error(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "res_dict = pre_gs_inst.cv_results_\n",
    "\n",
    "results_df = pd.DataFrame()\n",
    "for key in res_dict.keys():\n",
    "     results_df[key] = res_dict[key]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_min_samples_leaf</th>\n",
       "      <th>param_max_features</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>mean_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>sqrt</td>\n",
       "      <td>50</td>\n",
       "      <td>0.806128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>sqrt</td>\n",
       "      <td>50</td>\n",
       "      <td>0.804887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>sqrt</td>\n",
       "      <td>50</td>\n",
       "      <td>0.790066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11</td>\n",
       "      <td>sqrt</td>\n",
       "      <td>50</td>\n",
       "      <td>0.777692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>log2</td>\n",
       "      <td>50</td>\n",
       "      <td>0.807654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2</td>\n",
       "      <td>log2</td>\n",
       "      <td>50</td>\n",
       "      <td>0.808955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>log2</td>\n",
       "      <td>50</td>\n",
       "      <td>0.794156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>11</td>\n",
       "      <td>log2</td>\n",
       "      <td>50</td>\n",
       "      <td>0.789136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>0.793475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>0.794628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>0.786672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>0.779913</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   param_min_samples_leaf param_max_features param_n_estimators  \\\n",
       "0                       1               sqrt                 50   \n",
       "1                       2               sqrt                 50   \n",
       "2                       7               sqrt                 50   \n",
       "3                      11               sqrt                 50   \n",
       "4                       1               log2                 50   \n",
       "5                       2               log2                 50   \n",
       "6                       7               log2                 50   \n",
       "7                      11               log2                 50   \n",
       "8                       1                  1                 50   \n",
       "9                       2                  1                 50   \n",
       "10                      7                  1                 50   \n",
       "11                     11                  1                 50   \n",
       "\n",
       "    mean_test_score  \n",
       "0          0.806128  \n",
       "1          0.804887  \n",
       "2          0.790066  \n",
       "3          0.777692  \n",
       "4          0.807654  \n",
       "5          0.808955  \n",
       "6          0.794156  \n",
       "7          0.789136  \n",
       "8          0.793475  \n",
       "9          0.794628  \n",
       "10         0.786672  \n",
       "11         0.779913  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_cols = [x for x in results_df.columns if 'param' in x and x is not 'params']\n",
    "focus_cols = param_cols + ['mean_test_score']\n",
    "\n",
    "results_df[focus_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('param_min_samples_leaf', 1, 0.8024192191191825, 0.007783241825132443)\n",
      "('param_min_samples_leaf', 2, 0.8028235325041541, 0.00738317306417046)\n",
      "('param_min_samples_leaf', 7, 0.7902980290838647, 0.003747006566392743)\n",
      "('param_min_samples_leaf', 11, 0.7822471901387741, 0.006068451223665355)\n",
      "('param_max_features', 1.0, 0.7886721884221682, 0.006813342986028812)\n",
      "('param_max_features', 'log2', 0.7999754116227418, 0.009848313094719365)\n",
      "('param_max_features', 'sqrt', 0.7946933780895719, 0.013479682396875855)\n",
      "('param_n_estimators', 50, 0.794446992711494, 0.010579689791395532)\n"
     ]
    }
   ],
   "source": [
    "for param in param_cols:\n",
    "    for key,group in results_df.groupby(param):\n",
    "        print (param, key, group['mean_test_score'].mean(),group['mean_test_score'].std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jdavila\\Anaconda27\\lib\\site-packages\\sklearn\\model_selection\\_search.py:747: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[mean: 0.80813, std: 0.00630, params: {'n_estimators': 100, 'max_features': 'sqrt', 'oob_score': True, 'min_samples_leaf': 2},\n",
       " mean: 0.80653, std: 0.00757, params: {'n_estimators': 50, 'max_features': 'log2', 'oob_score': True, 'min_samples_leaf': 1},\n",
       " mean: 0.80570, std: 0.00536, params: {'n_estimators': 50, 'max_features': 'sqrt', 'oob_score': True, 'min_samples_leaf': 2},\n",
       " mean: 0.80425, std: 0.00839, params: {'n_estimators': 100, 'max_features': 'sqrt', 'oob_score': True, 'min_samples_leaf': 3},\n",
       " mean: 0.79684, std: 0.00838, params: {'n_estimators': 100, 'max_features': 'log2', 'oob_score': False, 'min_samples_leaf': 7},\n",
       " mean: 0.79654, std: 0.00659, params: {'n_estimators': 50, 'max_features': 'log2', 'oob_score': True, 'min_samples_leaf': 7},\n",
       " mean: 0.79496, std: 0.01147, params: {'n_estimators': 50, 'max_features': 1.0, 'oob_score': True, 'min_samples_leaf': 2},\n",
       " mean: 0.79345, std: 0.01063, params: {'n_estimators': 50, 'max_features': 1.0, 'oob_score': True, 'min_samples_leaf': 3},\n",
       " mean: 0.79194, std: 0.01053, params: {'n_estimators': 50, 'max_features': 1.0, 'oob_score': False, 'min_samples_leaf': 1},\n",
       " mean: 0.78773, std: 0.00908, params: {'n_estimators': 50, 'max_features': 'log2', 'oob_score': False, 'min_samples_leaf': 11},\n",
       " mean: 0.78665, std: 0.01322, params: {'n_estimators': 50, 'max_features': 1.0, 'oob_score': False, 'min_samples_leaf': 7},\n",
       " mean: 0.78164, std: 0.00493, params: {'n_estimators': 50, 'max_features': 'sqrt', 'oob_score': True, 'min_samples_leaf': 11},\n",
       " mean: 0.78133, std: 0.00770, params: {'n_estimators': 100, 'max_features': 'sqrt', 'oob_score': False, 'min_samples_leaf': 11},\n",
       " mean: 0.78132, std: 0.00720, params: {'n_estimators': 100, 'max_features': 'sqrt', 'oob_score': True, 'min_samples_leaf': 11},\n",
       " mean: 0.78071, std: 0.01551, params: {'n_estimators': 50, 'max_features': 1.0, 'oob_score': True, 'min_samples_leaf': 11}]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "\n",
    "param_dist = {'max_features' : ['sqrt','log2',1.0], \n",
    " 'min_samples_leaf' : [1, 2, 3, 7, 11],\n",
    " 'n_estimators': [50, 100],\n",
    " 'oob_score': [True, False]}\n",
    "\n",
    "pre_gs_inst = RandomizedSearchCV(RandomForestRegressor(warm_start=True), \n",
    " param_distributions = param_dist,\n",
    " cv=3, \n",
    " n_iter = 15)\n",
    "\n",
    "pre_gs_inst.fit(X_train, y_train)\n",
    "\n",
    "sorted(pre_gs_inst.grid_scores_, key = lambda x:x.mean_validation_score, reverse = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
       "           max_features='sqrt', max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "           min_samples_leaf=2, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, n_estimators=1000, n_jobs=-1,\n",
       "           oob_score=True, random_state=None, verbose=0, warm_start=True)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rs_rft = RandomForestRegressor(n_jobs=-1, warm_start=True,\n",
    " max_features = 'sqrt',\n",
    " min_samples_leaf = 2,\n",
    " oob_score = True,\n",
    " n_estimators = 1000)\n",
    "rs_rft.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.81772579447544591"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = rs_rft.predict(X_test)\n",
    "from sklearn.metrics import r2_score\n",
    "r2_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R-squared 0.817725794475\n",
      "MAE :  0.330175719471\n",
      "MAPE :  0.188636567659\n"
     ]
    }
   ],
   "source": [
    "y_pred = rs_rft.predict(X_test)\n",
    "\n",
    "from sklearn.metrics import r2_score, mean_absolute_error\n",
    "\n",
    "print \"R-squared\",r2_score(y_test, y_pred)\n",
    "print \"MAE : \",mean_absolute_error(y_test, y_pred)\n",
    "print \"MAPE : \",(np.abs(y_test - y_pred)/y_test).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_features': 'log2',\n",
       " 'min_samples_leaf': 1,\n",
       " 'n_estimators': 50,\n",
       " 'oob_score': False}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import ExtraTreesRegressor\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "\n",
    "\n",
    "param_dist = {'max_features' : ['sqrt','log2',1.0],\n",
    "'min_samples_leaf' : [1, 2, 3, 7, 11],\n",
    "'n_estimators': [50, 100],\n",
    "'oob_score': [True, False]}\n",
    "\n",
    "pre_gs_inst = RandomizedSearchCV(ExtraTreesRegressor(warm_start=True,bootstrap=True),\n",
    "param_distributions = param_dist,\n",
    "cv=3,\n",
    "n_iter = 15)\n",
    "\n",
    "pre_gs_inst.fit(X_train, y_train)\n",
    "\n",
    "pre_gs_inst.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import copy\n",
    "\n",
    "extra_best_params = copy.deepcopy(pre_gs_inst.best_params_)\n",
    "\n",
    "extra_best_params['n_estimators'] = 1000\n",
    "extra_best_params['warm_start'] = True\n",
    "extra_best_params['bootstrap'] = True\n",
    "\n",
    "rs_etr = ExtraTreesRegressor(**extra_best_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ExtraTreesRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
       "          max_features='log2', max_leaf_nodes=None,\n",
       "          min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "          min_samples_leaf=1, min_samples_split=2,\n",
       "          min_weight_fraction_leaf=0.0, n_estimators=1000, n_jobs=1,\n",
       "          oob_score=False, random_state=None, verbose=0, warm_start=True)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rs_etr.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.81995468995123055"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = rs_etr.predict(X_test)\n",
    "from sklearn.metrics import r2_score\n",
    "r2_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R-squared 0.819954689951\n",
      "MAE :  0.326634543909\n",
      "MAPE :  0.184605707508\n"
     ]
    }
   ],
   "source": [
    "y_pred = rs_etr.predict(X_test)\n",
    "\n",
    "from sklearn.metrics import r2_score, mean_absolute_error\n",
    "\n",
    "print \"R-squared\",r2_score(y_test, y_pred)\n",
    "print \"MAE : \",mean_absolute_error(y_test, y_pred)\n",
    "print \"MAPE : \",(np.abs(y_test - y_pred)/y_test).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#In Linux command line: install pyearth\n",
    "git clone git://github.com/scikit-learn-contrib/py-earth.git\n",
    "cd py-earth\n",
    "sudo python setup.py install"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyearth import Earth\n",
    "earth_model = Earth().fit(X_train, y_train)\n",
    "\n",
    "from sklearn.metrics import r2_score\n",
    "y_pred = earth_model.predict(X_test)\n",
    "r2_score(y_test, y_pred)\n",
    "\n",
    "# 0.70532965099191069  #: In Linux"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import BayesianRidge\n",
    "earth_pipeline = Pipeline([('earth', Earth()), ('br',BayesianRidge())]).fit(X_train, y_train)\n",
    "y_pred = earth_pipeline.predict(X_test)\n",
    "r2_score(y_test, y_pred)\n",
    "# 0.70538218833022714  #: In Linux"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
